#!/usr/bin/env python2.7
# -*- coding: utf-8 -*-
#
#   Gmail Backup library
#   
#   Copyright © 2008, 2009, 2010 Jan Svec <honza.svec@gmail.com> and Filip Jurcicek <filip.jurcicek@gmail.com>.
#
#   This file is part of Gmail Backup.
#
#   Gmail Backup is free software: you can redistribute it and/or modify it
#   under the terms of the GNU General Public License as published by the Free
#   Software Foundation, either version 3 of the License, or (at your option)
#   any later version.
#
#   Gmail Backup is distributed in the hope that it will be useful, but WITHOUT
#   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
#   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
#   more details.
#
#   You should have received a copy of the GNU General Public License along
#   with Gmail Backup.  If not, see <http://www.gnu.org/licenses/
#
#   See LICENSE file for license details
#
#   With changes by Jesper Hertel <jesper.hertel@gmail.com> (JH).

# Standard modules.
import codecs
import imaplib
import os
import re
import shutil
import socket
import string
import sys
import time
import traceback
import unicodedata
import urllib
import zipfile

# Gmail Backup modules.
from messagefunctions import _trimDateTimeTuple, _getMessageDateAsDateTimeTuple,\
    _getSenderAndSubjectFromMessageObject, getDateTimeTupleFromMessageAsString,\
    _getSenderAndSubjectFromMessageAsString, _getMessageIdFromMessageAsString,\
    _getMessageObjectFromMessageAsString, _getMessageIdFromMessageObject,\
    getImap4InternalDateStringFromMessageAsString
import DateTimeConverter
import GmailConnection
import messagefunctions
import translation

_submoduleListForVersioning = [DateTimeConverter, GmailConnection, messagefunctions, translation]
import versioning; _revisionInfo = versioning.getRevisionInfoFromTagsAndSubModules('$Revision$', '$Date$', _submoduleListForVersioning)

GMB_REVISION = _revisionInfo.revisionNumber  # E.g. 25.
GMB_DATE_ISO_FORMAT = _revisionInfo.isoDate  # E.g. "2013-07-16".

SPEED_AVERAGE_TIME_MINIMUM_SECONDS = 21 # speed average over the last this many seconds

MAX_LABEL_RETRIES = 5  # Maximum number of retries for downloading messages for a given label.

VERSION_URL = 'http://code.google.com/p/gmail-backup-com/source/list'
DOWNLOAD_URL = 'http://code.google.com/p/gmail-backup-com/downloads/list'

BYTES_PER_KILOBYTE_FLOAT = 1024.
BYTES_PER_MEGABYTE_FLOAT = 1024. * 1024.
ONE_DAY_IN_SECONDS = 24*60*60

translation.installTranslation()



def _revertDict(theDictionary):
    '''Return a dictionary where all keys and values in theDictionary are swapped.'''
    return dict((v, k) for (k, v) in theDictionary.iteritems())


def _shiftDateTimeTupleIfPossible(dateTimeTuple, adjustmentInSeconds):
    secondsSinceTheEpoch = time.mktime(dateTimeTuple)
    try:
        # Try to adjust.
        resultingStructTime = time.localtime(secondsSinceTheEpoch + adjustmentInSeconds)
    except ValueError:
        # Adjusting gave an error. Then we don't adjust.
        resultingStructTime = time.localtime(secondsSinceTheEpoch)
    return resultingStructTime  # Which is also a DateTimeTuple.


def _shiftAndTrimDateTimeTupleIfPossible(dateTimeTuple, adjustmentInSeconds):

    # Trim the date to a valid date.
    # JH: Why?
    trimmedDateTimeTuple = _trimDateTimeTuple(dateTimeTuple)

    # Try to shift the date adjustmentInSeconds seconds forward.
    # adjustmentInSeconds can be negative.
    possiblyShiftedDateTimeTuple = _shiftDateTimeTupleIfPossible(trimmedDateTimeTuple, adjustmentInSeconds)
    
    # Trim the resulting date to a valid date 
    # JH: Why? It is a valid time.struct_time created by time.localtime().
    possiblyShiftedAndTrimmedDateTimeTuple = _trimDateTimeTuple(possiblyShiftedDateTimeTuple)
    
    # Return the possibly shifted and trimmed date.
    return possiblyShiftedAndTrimmedDateTimeTuple


def _shiftAndTrimDateTimeTuplesIfPossible(firstDateTimeTuple, lastDateTimeTuple):
    '''Return a 2-tuple (pair) of the given dateTimeTuples where the first 
    dateTimeTuple in the result is one day before the given 
    firstDateTimeTuple and the last dateTimeTuple in the result is one 
    day after the given lastDateTimeTuple.
    
    Both dates are also adjusted to be valid dates between the years 1970 
    and 9999 (in _trimDateTimeTuple()). JH: Why?
    
    JH: I think the meaning of this function is to adjust for time zones and
    be sure that you get all emails within a local date interval even if the 
    date in the email is one day before or after the date searched for.
    
    JH: Why it is adjusted to be not before January 1st, 1970, must be so 
    the dates can be converted to the 'seconds since the system Epoch' time
    format. Whether this is absolutely necessary, or if the 'seconds since 
    the system Epoch' time format can be avoided, I don't know. But maybe emails 
    cannot have been sent before 1970 (which is the system Epoch on Unix)? 
    When was email invented? Well, according to 
    https://en.wikipedia.org/wiki/Email, the first standards for encoding
    email are from 1973. So dates before 1973 should really not appear in 
    emails as time stamps.
    '''

    possiblyOneDayBeforeFirstDateTimeTuple = _shiftAndTrimDateTimeTupleIfPossible(firstDateTimeTuple, -ONE_DAY_IN_SECONDS)
    possiblyOneDayAfterLastDateTimeTuple = _shiftAndTrimDateTimeTupleIfPossible(lastDateTimeTuple, ONE_DAY_IN_SECONDS)
    
    # Return the resulting and trimmed dateTimeTuples.
    return possiblyOneDayBeforeFirstDateTimeTuple, possiblyOneDayAfterLastDateTimeTuple


def imap_decode(stringCodedInImapUtf7):
    '''
    Decode the modified utf-7 that IMAP uses (cf. http://www.faqs.org/rfcs/rfc1642.html).
 
    Returns:
        A decoded string (not unicode).
    '''

    # JH 2013-07-19: I am unsure if this is correctly coded. 
    # It seems to be too simple to be correct if you compare with 
    # RFC 1642 (http://www.faqs.org/rfcs/rfc1642.html).

    def replacePart(matchObject):

        matchedString = matchObject.groups(1)[0] # Get what was between "&" and "-" into matchedString.

        if matchedString:
            # Something was between "&" and "-".
            s = ('+' + matchedString + '-').replace(',', '/')  # In modified utf-7, "," is used instead of "/".
            decodedPart = s.decode('utf-7')
        else:
            # Nothing between "&" and "-", i.e. "&-" which is the encoding for "&". 
            decodedPart = '&'

        return decodedPart
    
    return re.sub('&(.*?)-', replacePart, stringCodedInImapUtf7)


def imap_encode(s):
    '''
    Decode the string s to the modified utf-7 that IMAP uses 
    (cf. http://www.faqs.org/rfcs/rfc1642.html).
    '''

    # JH 2013-07-19: I am unsure if this is correctly coded. 
    # It seems too simple to be correct if you compare with RFC 1642 
    # (http://www.faqs.org/rfcs/rfc1642.html).
    # Also, what is with the ".encode('ascii', 'replace')"? Why
    # was that found necessary? What was found needed to be thrown away 
    # by the 'replace'?

    def replacePart(matchObject):
        matchedString = matchObject.groups(1)[0]
        if matchedString == '&':
            encodedPart = '&-'  # "The character "&" (0x26) is represented by the two-octet sequence "&-"." (http://www.faqs.org/rfcs/rfc1642.html)
        else:
            encodedPart = matchedString.encode('utf-7').replace('+', '&').replace('/', ',')
        return encodedPart
    
    return re.sub('([^\x20-\x25\x27-\x7e]+|&)', replacePart, s).encode('ascii', 'replace')


def _removeDiacritics(s):
    '''Return `s` with any diacritics removed.
    
    :param s:
        A unicode or non-unicode string.
        
    Doctests:
        We can't make doctests as the Nose tool doesn't handle 
        character sets in docstrings well. Therefore we use the 
        specific test_removeDiacritics() function below instead.
    '''
    
    def isNotCombiningCharacter(unicodeCharacter):
        return not unicodedata.combining(unicodeCharacter)
    
    unicodeString = unicode(s)  # This works even if s is already a unicode string.

    normalizedUnicodeString = unicodedata.normalize('NFKD', unicodeString)

    # Remove all combining characters (but not the letters they attach to).
    # In this way, "é" is replaced with "e", etc.
    filteredUnicodeString = filter(isNotCombiningCharacter, normalizedUnicodeString)

    return filteredUnicodeString

def test_removeDiacritics():
    '''

    Doctests:
        >>> test_removeDiacritics()
    '''
    assert _removeDiacritics(u"Père Noël") == u'Pere Noel'
    assert _removeDiacritics(u"Åleflæsk") == u'Aleflæsk'
    assert _removeDiacritics(u"""ò ó ô õ ō o̅ ŏ ȯ ö ỏ o̊ ő ǒ o̍ o̎ ȍ o̐ ȏ o̒ o̓ o̔ o̕ o̖ o̗ o̘ o̙ o̚ ơ o̜ o̝ o̞ o̟ o̠ o̡ o̢ ọ o̤ o̥ o̦ o̧ ǫ o̩ o̪ o̫ o̬ o̭ o̮ o̯ o̰ o̱ o̲ o̳ o̴ o̵ o̶ o̷ o̸ o̹ o̺ o̻ o̼ o̽ o̾ o̿ ò ó o͂ o̓ ö́ oͅ o͆ o͇ o͈ o͉ o͊ o͋ o͌ o͍ o͎ o͏ o͐ o͑ o͒ o͓ o͔ o͕ o͖ o͗ o͘ o͙ o͚ o͛ o͜a o͝a o͞a o͟a o͠a o͡a o͢a oͣ oͤ oͥ oͦ oͧ oͨ oͩ oͪ oͫ oͬ oͭ oͮ oͯ""") == u"o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o͏ o o o o o o o o o o o o oa oa oa oa oa oa oa o o o o o o o o o o o o o"


def getExtension(filename):
    '''Return the extension, if any, of the given filename, in the form ".ext".
    Return the empty string if the filename has no extension.
    
    Doctests:

        Normal cases:
        >>> getExtension("test.doc")
        '.doc'
        >>> getExtension("folder/test.doc")
        '.doc'
        >>> getExtension("folder/test.xls")
        '.xls'
        >>> getExtension(r"folder\test.txt")
        '.txt'
        >>> getExtension(r"C:\Data\My Folder.With Extension\test.py")
        '.py'
        >>> getExtension("test")
        ''
        
        Special cases:
        >>> getExtension(".ext")
        ''
        >>> getExtension("test.")
        '.'
        >>> getExtension(".test.")
        '.'
        >>> getExtension(r"C:\Data\My Folder\test.MyVeryLongExtension with spaces")
        '.MyVeryLongExtension with spaces'
        >>> getExtension(r"C:\Data\My Folder\test. MyEvenLongerExtension with more  spaces ")
        '. MyEvenLongerExtension with more  spaces '
    '''
    return os.path.splitext(filename)[1]

def fileExists(filename):
    '''Return True if a regular file (not a directory) with the given filename 
    exists, and False otherwise.'''
    return os.path.isfile(filename)


class GmailBackupNotifier(object):
    '''
    A base notifier class for passing notifications to the user. 
    
    This base class itself has method implementations that do nothing.
    
    Note that the class is not an abstract class, but rather contains 
    fall-through method implementations that do nothing. It is up to 
    derived classes to add functionality for the methods wanted.
    '''
    
    def notifyVersion(self):
        pass

    def notifySpeed(self, amount, timeDifference):
        pass

    def notifyBackup(self, end, mailbox, directory):
        pass

    def notifyRestore(self, end, mailbox, directory):
        pass

    def notifyClear(self, end, mailbox):
        pass

    def notifyEmailBackup(self, from_address, subject, num, total):
        pass

    def notifyEmailBackupSkip(self, num, total, skipped, total_to_skip):
        pass

    def notifyEmailRestore(self, from_address, subject, num, total):
        pass

    def notifyEmailRestoreSkip(self, from_address, subject, num, total):
        pass

    def notifyLabelsBackup(self, processEnded):
        pass

    def notifyLabelsRestore(self, num, total):
        pass

    def notifyError(self, msg):
        pass
    
    def notifyLog(self, msg):
        pass

    def notifyException(self, etype, error, tb):
        pass

    def notifyExceptionFull(self, etype, error, tb):
        pass

    def notifyExceptionMessage(self, msg, etype, error, tb):
        pass

    def getCurrentAverageSpeedInKilobytesPerSecond(self):
        pass

    def getTotalProcessedInMegabytes(self):
        pass

    def getPercentageCompleted(self):
        pass

    def updateSpeedDisplay(self):
        pass


class GmailBackupConsoleNotifier(GmailBackupNotifier):
    FATAL_ERROR_CLASS_LIST = [socket.error, imaplib.IMAP4.abort, imaplib.IMAP4.error, KeyboardInterrupt]

    def __init__(self, *args, **kwargs):
        super(GmailBackupConsoleNotifier, self).__init__(*args, **kwargs)
        self._resetCounters()

    def _resetCounters(self):
        self._speedArray = []  # List of (numberOfBytes, durationInSeconds) pairs.
        self._totalBytes = 0  # Total amount of processed data (read or written) in the current process, measured in bytes.
        self._percentageCompleted = None

    def updatePrint(self, message):
        try:
            print message
        except UnicodeEncodeError:
            print message.encode('ascii', 'replace')
        sys.stdout.flush()

    def updatePrintTypeOver(self, message):
        if sys.stdout.isatty():
            try:
                print '\r%s    \r' % (message),
            except UnicodeEncodeError:
                print '\r%s    \r' % (message.encode('ascii', 'replace'), ),
            sys.stdout.flush()

    def notifyVersion(self):
        self.notifyLog(_("Gmail Backup revision %s (%s)") % (GMB_REVISION, GMB_DATE_ISO_FORMAT))

    def notifySpeed(self, newNumberOfBytes, newDurationInSeconds):
        self._totalBytes += newNumberOfBytes
        self._speedArray.insert(0, (newNumberOfBytes, newDurationInSeconds))
        durationInSecondsSum = 0
        for index, (numberOfBytes, durationInSeconds) in enumerate(self._speedArray):  # @UnusedVariable numberOfBytes
            durationInSecondsSum += durationInSeconds
            if durationInSecondsSum > SPEED_AVERAGE_TIME_MINIMUM_SECONDS:
                break
        del self._speedArray[index+1:]  # Delete recordings for more than SPEED_AVERAGE_TIME_MINIMUM_SECONDS seconds ago.
        self.updateSpeedDisplay()

    def getCurrentAverageSpeedInKilobytesPerSecond(self):
        '''Return average speed in kilobytes per second over the last minimum SPEED_AVERAGE_TIME_MINIMUM_SECONDS.'''
        totalNumberOfBytes = 0
        totalDurationInSeconds = 0
        for numberOfBytes, durationInSeconds in self._speedArray:
            totalNumberOfBytes += numberOfBytes
            totalDurationInSeconds += durationInSeconds
        if totalDurationInSeconds == 0:
            averageSpeedInKilobytesPerSecond = 0
        else:
            averageSpeedInKilobytesPerSecond = (totalNumberOfBytes / totalDurationInSeconds) / BYTES_PER_KILOBYTE_FLOAT
        
        return averageSpeedInKilobytesPerSecond
    
    def getTotalProcessedInMegabytes(self):
        '''Return a floating point number with the total amount of processed data measured in megabytes.'''
        return self._totalBytes/BYTES_PER_MEGABYTE_FLOAT

    def getPercentageCompleted(self):
        return self._percentageCompleted

    def updateSpeedDisplay(self):
        self.updatePrintTypeOver(_("%1.f KB/s (total: %.2f MB)") % (self.getCurrentAverageSpeedInKilobytesPerSecond(), self.getTotalProcessedInMegabytes()))

    def notifyBackup(self, processEnded, username, filenamePattern):
        '''Called at the beginning (processEnded=False) and end 
        (processEnded=True) of a backup process.'''
        if not processEnded:
            self._resetCounters()
            self.updatePrint(_("Starting backup of account %s into %s") % (username, filenamePattern))
        else:
            self.updatePrint(_("Ending backup of account %s") % (username, ))

    def notifyRestore(self, processEnded, username, filenamePattern):
        '''Called at the beginning (processEnded=False) and end 
        (processEnded=True) of a restore process.'''
        if not processEnded:
            self._resetCounters()
            self.updatePrint(_("Restoring the content of account %s from %s") % (username, filenamePattern))
        else:
            self.updatePrint(_("End of restoring of account %s") % (username, ))

    def notifyClear(self, processEnded, username):
        if not processEnded:
            self.updatePrint(_("Clearing the content of account %s") % (username, ))
        else:
            self.updatePrint(_("End of clearing of account %s") % (username, ))

    def _updatePercentageCompleted(self, numberOfMessagesHandled, totalNumberOfMessages):
        self._percentageCompleted = float(numberOfMessagesHandled) / totalNumberOfMessages * 100

    def notifyEmailBackup(self, messageSender, messageSubject, numberOfMessagesHandled, totalNumberOfMessages):
        self._updatePercentageCompleted(numberOfMessagesHandled, totalNumberOfMessages)
        self.updatePrint(
            _("Stored %4.1f%% (%d of %d): %s - %s") % (
            self._percentageCompleted, 
            numberOfMessagesHandled, 
            totalNumberOfMessages, 
            messageSender, 
            messageSubject))

    def notifyEmailBackupSkip(self, numberOfMessagesHandled, totalNumberOfMessages, skipped, total_to_skip):
        self._updatePercentageCompleted(numberOfMessagesHandled, totalNumberOfMessages)
        self.updatePrint(_("Skip   %4.1f%% (%d of %d): message already stored") % (self._percentageCompleted, numberOfMessagesHandled, totalNumberOfMessages))

    def notifyEmailRestore(self, messageSender, messageSubject, numberOfMessagesHandled, totalNumberOfMessages):
        self._updatePercentageCompleted(numberOfMessagesHandled, totalNumberOfMessages)
        self.updatePrint(_("Restored %4.1f%%: %s - %s") % (self._percentageCompleted, messageSender, messageSubject))

    def notifyEmailRestoreSkip(self, messageSender, messageSubject, numberOfMessagesHandled, totalNumberOfMessages):
        self._updatePercentageCompleted(numberOfMessagesHandled, totalNumberOfMessages)
        self.updatePrint(_("Skipdate %4.1f%%: %s - %s") % (self._percentageCompleted, messageSender, messageSubject))

    def notifyLabelsBackup(self, processEnded):
        if not processEnded:
            self.updatePrint(_("Starting backup of labels"))
        else:
            self.updatePrint(_("Backup of labels ended"))

    def notifyLabelsRestore(self, num, total):
        self.updatePrint(_("Restoring labels, %.1f%%") % (float(num) / total * 100,))

    def notifyError(self, errorMessage):
        self.updatePrint(_('Error: %s') % errorMessage)

    def notifyLog(self, logMessage):
        self.updatePrint(unicode(logMessage))

    def formatException(self, exceptionType, exception, exceptionTraceback):
        return ''.join(traceback.format_exception(exceptionType, exception, exceptionTraceback))

    def notifyException(self, exceptionType, exception, exceptionTraceback):
        if isinstance(exception, socket.error):
            #num, message = exception
            self.notifyError(_('%s\nThere are network problems. Please try again later.') % (exception, ))
        elif isinstance(exception, imaplib.IMAP4.abort):
            self.notifyError(_("IMAP aborted the transfer"))
        elif isinstance(exception, imaplib.IMAP4.error):
            self.notifyError(_("IMAP: %s") % exception.message)
        elif isinstance(exception, KeyboardInterrupt):
            self.notifyLog(_("Program interrupted by user"))
        else:
            self.notifyExceptionFull(exceptionType, exception, exceptionTraceback)

    def notifyExceptionFull(self, exceptionType, exception, exceptionTraceback):
        formattedException = self.formatException(exceptionType, exception, exceptionTraceback)
        self.notifyError(
                         _("Error occurred, full traceback is below (gmb.py revision %s)\n%s") 
                         % (GMB_REVISION, formattedException))

    def notifyExceptionMessage(self, message, exceptionType, exception, exceptionTraceback):
        formattedException = self.formatException(exceptionType, exception, exceptionTraceback)
        self.notifyError(
                         _("%s\n" \
                           "Ignoring this exception message.\n" \
                           "If you want, please report the following traceback to gmail-backup-com-users@googlegroups.com\n" \
                           "Thanks! (gmb.py revision %s):\n" \
                           "%s") 
                         % (message, GMB_REVISION, formattedException))

    def notifyLastException(self):
        exceptionType, exception, exceptionTraceback = sys.exc_info()
        try:
            self.notifyException(exceptionType, exception, exceptionTraceback)
        finally:
            del exceptionTraceback  # "Make sure to delete it after use" – http://docs.python.org/2.7/library/sys


    def isFatalException(self, exception):
        def isInstanceOfOneOfClasses(theObject, classList):
            for classObject in classList:
                if isinstance(theObject, classObject):
                    return True
            return False

        return isInstanceOfOneOfClasses(exception, self.FATAL_ERROR_CLASS_LIST)
    
    def handleException(self, message):
        exceptionType, exception, exceptionTraceback = sys.exc_info()
        
        try:
            if not self.isFatalException(exception):
                # The exception was not fatal. Just show a message about the 
                # error to the user (in the log), and then allow the process
                # to continue.
                self.notifyExceptionMessage(message, exceptionType, exception, exceptionTraceback)
            else:
                # The exception is fatal. Re-raise the exception.
                raise
        finally:
            del exceptionTraceback  # "Make sure to delete it after use" – http://docs.python.org/2.7/library/sys



class EmailStorage(object):

    @classmethod
    def createStorage(cls, folderAndFileTemplate, notifier):
        '''
        Factory method.
        
        :param cls: 
            The EmailStorage class.
        :param folderAndFileTemplate:
            A string with a template for the directory and file name pattern
            for stored email files.
        :param notifier:
            A GmailBackupNotifier object.
        '''
        templateExtension = getExtension(folderAndFileTemplate.split('#')[0])
        if templateExtension.lower() == '.zip':
            emailStorage = ZipStorage(folderAndFileTemplate, notifier)
        else:
            emailStorage = DirectoryStorage(folderAndFileTemplate, notifier)
        return emailStorage

    def getSetOfDownloadedMessageIds(self):
        '''Returns the set of stored msg_ids'''
        raise NotImplementedError()  # The method is abstract and should not be called.

    def iterateBackupFiles(self, since_time=None, before_time=None, logging=True):
        '''Iterates over backups specified by parameters and yields pairs (storageid, message)'''
        raise NotImplementedError()

    def storeMessage(self, messageAsString, messageObject):
        '''Stores the given message.'''
        raise NotImplementedError()

    def getCopyOfMessageIdToLabelsDict(self):
        '''Returns label assignment'''
        raise NotImplementedError()

    def updateLabelAssignment(self, assignment):
        '''Updates label assignment with `assignment`'''
        raise NotImplementedError()

    def newestBackedUpEmailDateTimeTuple(self):
        '''Return a dateTimeTuple with the date of the newest of all backed up emails.'''
        raise NotImplementedError()

    def updateStamp(self, last_time):
        '''Updates the stamp of the last backup to last_time'''
        raise NotImplementedError()

    def cleanTemplateDictValues(self, templateDict):
        '''Replace "/" and "\" (directory separators) with "_" in all values 
        in the dictionary templateDict.
        We can't do this in _cleanFilename() as the filename being cleaned 
        there can include valid directory separators, so we have to clean
        it in the template values themselves.'''
        def cleanTemplateValue(value):
            return value.replace('/', '_').replace('\\', '_')
        
        return dict((k, cleanTemplateValue(v)) for (k, v) in templateDict.iteritems())

    def _getTemplateDictionary(self, messageObject):
        '''Create dictionary used in the template expansion.'''
        templateDict = {}

        dateTimeTuple = _getMessageDateAsDateTimeTuple(messageObject)
        templateDict['YEAR'] = time.strftime('%Y', dateTimeTuple)
        templateDict['MONTH'] = time.strftime('%m', dateTimeTuple)
        templateDict['DAY'] = time.strftime('%d', dateTimeTuple)
        templateDict['HOUR'] = time.strftime('%H', dateTimeTuple)
        templateDict['MINUTE'] = time.strftime('%M', dateTimeTuple)
        templateDict['SECOND'] = time.strftime('%S', dateTimeTuple)

        sender, subject = _getSenderAndSubjectFromMessageObject(messageObject)
        templateDict['FROM'] = sender.lower()
        templateDict['SUBJ'] = subject
        templateDict['SUBJECT'] = subject  # JH: Added.
        
        cleanedTemplateDict = self.cleanTemplateDictValues(templateDict)
        return cleanedTemplateDict


class DirectoryStorage(EmailStorage):

    def __init__(self, folderAndFileTemplate, notifier):
        self.setBaseDirectoryAndFragment(folderAndFileTemplate)
        self.notifier = notifier
        self._makeMaildir()
        self._readDownloadedIds()
        self._readLabelAssignment()

    DEFAULT_TEMPLATE_FRAGMENT = '${YEAR}/${MONTH}/${YEAR}${MONTH}${DAY}-${HOUR}${MINUTE}${SECOND}-${FROM}-${SUBJECT}'

    def setBaseDirectoryAndFragment(self, folderAndFileTemplate):
        '''Sets the baseDirectory and the template fragment for naming 
        the files in the storage.'''
        itemList = folderAndFileTemplate.split("#", 1)
        baseDirectoryPart = itemList[0]
        if len(itemList) == 1:
            # There were no "#" in folderAndFileTemplate.
            templateFragmentPart = self.DEFAULT_TEMPLATE_FRAGMENT
        else:
            templateFragmentPart = itemList[1]
        self.baseDirectory = os.path.expanduser(baseDirectoryPart)
        self.templateFragment = string.Template(templateFragmentPart)

    def iterateBackupFiles(self, sinceSecondsSinceTheEpoch=None, beforeSecondsSinceTheEpoch=None, logging=True):

        def walkBackups(directoryTop):
            '''Walk through the given directory, directoryTop, and yield paths 
            relative to directoryTop for all files ending with '.eml'.'''
            for pathToDirectory, directoryNameList, filenameList in os.walk(directoryTop):  # @UnusedVariable directoryNameList
                relativeDirectoryName = pathToDirectory[len(directoryTop):].lstrip(os.path.sep)
                for filename in filenameList:
                    if getExtension(filename).lower() == '.eml':
                        yield os.path.join(relativeDirectoryName, filename)

        listing = sorted(walkBackups(self.baseDirectory))
        for index, messageRelativeFilename in enumerate(listing):
            try:
                messageFullFilename = os.path.join(self.baseDirectory, messageRelativeFilename)
                with file(messageFullFilename, 'rb') as messageFileHandle:
                    messageAsString = messageFileHandle.read()

                messageDateTimeAsDateTimeTuple = getDateTimeTupleFromMessageAsString(messageAsString)
                messageDateTimeInSecondsSinceTheEpoch = time.mktime(messageDateTimeAsDateTimeTuple)
                messageIsWithinRequestedTimePeriod = (
                    (
                        sinceSecondsSinceTheEpoch is None 
                        or sinceSecondsSinceTheEpoch < messageDateTimeInSecondsSinceTheEpoch
                    ) and (
                        beforeSecondsSinceTheEpoch is None
                        or messageDateTimeInSecondsSinceTheEpoch < beforeSecondsSinceTheEpoch
                    ))

                if messageIsWithinRequestedTimePeriod:
                    yield messageRelativeFilename, messageAsString
                
                if logging:
                    messageSender, messageSubject = _getSenderAndSubjectFromMessageAsString(messageAsString)
                    if messageIsWithinRequestedTimePeriod:
                        self.notifier.notifyEmailRestore(messageSender, messageSubject, index+1, len(listing))
                    else:
                        self.notifier.notifyEmailRestoreSkip(messageSender, messageSubject, index+1, len(listing))
            except (GeneratorExit, ):
                break
            except:
                self.notifier.handleException(_("Error occurred while reading email from disk"))

    def _makeMaildir(self):
        directoryList = [
                         self.baseDirectory,
#                          os.path.join(self.baseDirectory, 'cur'),  # JH: What are these three cur, new, tmp for? They seem unused. Therefore I have uncommented them.
#                          os.path.join(self.baseDirectory, 'new'),
#                          os.path.join(self.baseDirectory, 'tmp')
                        ]
        for directory in directoryList:
            try:
                os.makedirs(directory)
            except OSError:
                pass

    def getIdCacheFilename(self):
        return os.path.join(self.baseDirectory, 'ids.txt')

    def getLabelsFilename(self):
        return os.path.join(self.baseDirectory, 'labels.txt')

    def getStampFilename(self):
        return os.path.join(self.baseDirectory, 'stamp')


    def _readMessageIdToMessageFilenameFromCache(self):
        '''Read the contents of the file with filename 
        self.getIdCacheFilename() into self.messageIdToMessageFilename.'''
        self.messageIdToMessageFilename = {}
        with file(self.getIdCacheFilename(), 'r') as idCacheFileHandle:
            for line in idCacheFileHandle:
                try: 
                    items = line.strip().split(None, 1) # Split on first whitespace and remove ending newline.
                    messageFilename = items[0]
                    messageId = items[1]
                    self.messageIdToMessageFilename[messageId] = messageFilename
                except:
                    # There was a syntax error in the ids.txt file.
                    # This should not be able to happen unless the file was edited manually.
                    # JH: Maybe this should trigger a re-caching of all files from the file content?
                    self.notifier.handleException(_("Bad line in file with cached MessageIDs: %s") % line.strip())

        self.messageFilenameToMessageId = _revertDict(self.messageIdToMessageFilename)

    def _readMessageIdToMessageFilenameFromMailFiles(self):
        '''Parse all backup email files and fill self.messageIdToMessageFilename 
        from the info in the files.'''
        self.messageIdToMessageFilename = {}
        self.messageFilenameToMessageId = {}
        for messageFilename, messageAsString in self.iterateBackupFiles(logging=False):
            try:
                messageId = _getMessageIdFromMessageAsString(messageAsString)
                self.messageIdToMessageFilename[messageId] = messageFilename
            except:
                self.notifier.handleException(_("Error while reading MessageID from stored message (%s)") % messageFilename)
        
        self.messageFilenameToMessageId = _revertDict(self.messageIdToMessageFilename)

    def _readDownloadedIds(self):
        '''Read self.messageIdToMessageFilename from either the cache file or the 
        backed up email files.'''
        if fileExists(self.getIdCacheFilename()):
            # The file ids.txt exists.
            # Read data into self.messageIdToMessageFilename from the ids.txt file.
            self.notifier.notifyLog(_("Reading messageIds from cache..."))
            self._readMessageIdToMessageFilenameFromCache()
        else:
            # The file ids.txt with cached ids of emails does not exist.
            self.notifier.notifyLog(_("Reading messageIds from stored files..."))
            self._readMessageIdToMessageFilenameFromMailFiles()

    def _writeDownloadedIds(self):
        '''Write all ids (self.messageIdToMessageFilename) to "ids.txt"'''
        idsFilename = self.getIdCacheFilename()
        if os.path.exists(idsFilename):
            os.remove(idsFilename)  # Delete already existing file. JH: Why? Doesn't it get overwritten when opened for writing?
        idsFileHandle = file(idsFilename, 'w')
        try:
            listOfMessageIdAndFilenamePairsSortedByFilename = sorted(self.messageIdToMessageFilename.items(), key=lambda item:item[1])
            for messageId, messageFilename in listOfMessageIdAndFilenamePairsSortedByFilename:
                try:
                    line = '%s\t%s' % (messageFilename, messageId)
                    print >> idsFileHandle, line
                except:
                    self.notifier.notifyError(_("Errorneous message in file: %s, please report it to <honza.svec@gmail.com>") % messageFilename)
        finally:
            idsFileHandle.close()

    def getSetOfDownloadedMessageIds(self):
        '''Return a set of all ids of all messages.'''
        return set(self.messageIdToMessageFilename)

    def getCopyOfMessageIdToLabelsDict(self):
        return self.messageIdToLabels.copy()

    def updateLabelAssignment(self, messageIdToLabelSetDict):
        self.messageIdToLabels.update(messageIdToLabelSetDict)
        self._writeLabelAssignment()


    def getGoodFilenameCharactersForCurrentOperatingSystem(self):
        '''
        
        JH: Note that the cleaning is different for different operating systems. This can be
        seen as a feature, but also as a potential problem, as some might backup the backups 
        themselves from one operating system to another (this has been done for real by at 
        least one person) and then run into trouble. Therefore I suggest to use the 
        lowest common denominator and only use characters that are safe in all major 
        operating systems.
        '''
        if os.name == 'posix': 
            # Linux, Unix, etc.
            goodFilenameCharacters = set('!"#\'()+-0123456789:;<=>@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]_abcdefghijklmnopqrstuvwxyz{}/\\')
        elif os.name == 'nt': 
            # Windows.
            goodFilenameCharacters = set("!#'()+-0123456789;=@ABCDEFGHIJKLMNOPQRSTUVWXYZ[]_abcdefghijklmnopqrstuvwxyz{}/\\")
        else:
            # All other operating systems.
            goodFilenameCharacters = set("+-0123456789=@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz/\\") 
        return goodFilenameCharacters

    def _cleanFilename(self, filename):
        '''
        Clean the filename - remove diacritics and other filesystem special characters.

        Returns:
            The cleaned filename.
        
        JH: Note that the cleaning is different for different operating systems. This can be
        seen as a feature, but also as a potential problem, as some might backup the backups 
        from one operating system to another and then run into trouble. Therefore I suggest
        to use the lowest common denominator and only use characters that are safe in all
        major operating systems.
        '''
        filenameWithoutDiacritics = _removeDiacritics(filename)
        filenameWithoutDiacriticsAndEncodedInUTF8 = filenameWithoutDiacritics.encode('utf-8', 'replace')
        goodFilenameCharacters = self.getGoodFilenameCharactersForCurrentOperatingSystem()
        accumulatedFilename = ''
        for character in filenameWithoutDiacriticsAndEncodedInUTF8:
            if character not in goodFilenameCharacters:
                character = '_'
            accumulatedFilename += character
        finalAccumulatedFilename = re.sub('_+', '_', accumulatedFilename) # Replace all sequences of "_" with a single "_".
        return finalAccumulatedFilename


    def getMessageFilename(self, messageAsString):
        messageObject = _getMessageObjectFromMessageAsString(messageAsString)
        templateDictionary = self._getTemplateDictionary(messageObject)
        uncleanedFilename = self.templateFragment.safe_substitute(templateDictionary)
        cleanedFilename = self._cleanFilename(uncleanedFilename)
        cleanedAndShortenedFilename = cleanedFilename[:100]  # Limit length of path. This is a partial fix / bettering of Gmail Backup Issue 18 (https://code.google.com/p/gmail-backup-com/issues/detail?id=18).
        return cleanedAndShortenedFilename
    
    def storeMessage(self, messageAsString, messageObject):
        messageFilename = self.getMessageFilename(messageAsString)
        messageDirectoryName = os.path.dirname(messageFilename)
        full_dn = os.path.join(self.baseDirectory, messageDirectoryName)
        if not os.path.isdir(full_dn):
            os.makedirs(full_dn)
        msg_iid = _getMessageIdFromMessageObject(messageObject)
        idx = 1
        while True:
            msg_fn_num = '%s-%01d.eml' % (messageFilename, idx)
            idx += 1
            full_fn_num = os.path.join(self.baseDirectory, msg_fn_num)
            if not os.path.exists(full_fn_num):
                break
        fw = file(full_fn_num, 'wb')
        try:
            fw.write(messageAsString)
        finally:
            fw.close()

        # Only store msg_fn_num and msg_iid if the writing to the file above succeeded.
        # This fixes a bug mentioned in Gmail Backup Issue 18 (https://code.google.com/p/gmail-backup-com/issues/detail?id=18)
        # where messages that failed to be written were never thereafter attempted to be downloaded again, and therefore
        # never being backed up again.
        self.messageIdToMessageFilename[msg_iid] = msg_fn_num
        self.messageFilenameToMessageId[msg_fn_num] = msg_iid

    def storeComplete(self):
        self._writeDownloadedIds()

    def _backupLabelAssignmentFile(self):
        labelsFilename = self.getLabelsFilename()
        labelsBackupFilename = self.getLabelsFilename()+'.bak'
        if os.path.exists(labelsFilename):
            if os.path.exists(labelsBackupFilename):
                os.remove(labelsBackupFilename)
            shutil.copy(labelsFilename, labelsBackupFilename)

    def _escapeLabels(self, labels):
        utf8_labels = [imap_decode(s) for s in labels]
        return ' '.join(s.replace('\t', '\\\t').replace(' ', '\\ ') for s in utf8_labels)

    def _unescapeLabels(self, string):
        lst = string.split()
        ret = []
        for i in lst:
            if ret and ret[-1][-1] == '\\':
                ret[-1] = ret[-1][:-1]+' '+i
            else:
                ret.append(i)
        ret = [imap_encode(s) for s in ret]
        return ret

    def _readLabelAssignment(self):
        labelsFilename = self.getLabelsFilename()
        self.messageIdToLabels = {}
        if os.path.isfile(labelsFilename):
            labelsFileHandle = codecs.open(labelsFilename, 'r', 'utf-8')  # Open the file for reading as utf-8.
            try:
                for line in labelsFileHandle:
                    items = line.split(None, 1)
                    messageFilename = items[0]
                    msg_iid = self.messageFilenameToMessageId.get(messageFilename, None)
                    if msg_iid is not None:
                        self.messageIdToLabels[msg_iid] = self._unescapeLabels(items[1])
            finally:
                labelsFileHandle.close()

    def _writeLabelAssignment(self):
        self._backupLabelAssignmentFile()
        fn = self.getLabelsFilename()
        if os.path.exists(fn):
            os.remove(fn)
        fw = codecs.open(fn, 'w', 'utf-8')
        for msg_iid, labels in sorted(self.messageIdToLabels.items()):
            msg_fn = self.messageIdToMessageFilename.get(msg_iid)
            if msg_fn is None:
                # We are unable do determine the filename for msg_iid
                continue
            print >> fw, '%s\t%s' % (msg_fn, self._escapeLabels(labels))
        fw.close()

    def newestBackedUpEmailDateTimeTuple(self):
        '''Return a dateTimeTuple with the date (not time) of the newest of all 
        backed up emails, or None if not available or if there is an error 
        reading the file with the date.'''
        stampFilename = self.getStampFilename()
        try:
            with file(stampFilename, 'r') as stampFileHandle:
                stampDateYYYYMMDD = stampFileHandle.readline().strip()  # The file contains e.g. "20130716" (without the quotes) for July 16, 2013.
        except IOError:
            stampDateYYYYMMDD = None

        if stampDateYYYYMMDD is not None:
            lastStampDateTimeTuple = time.strptime(stampDateYYYYMMDD, '%Y%m%d')
        else:
            lastStampDateTimeTuple = None
             
        return lastStampDateTimeTuple  # Might be None.

    def updateStamp(self, last_time):
        if last_time is None:
            return

        last_time = time.strftime('%Y%m%d', last_time)

        stampFile = self.getStampFilename()
        if os.path.exists(stampFile):
            os.remove(stampFile)
        fw = file(stampFile, 'w')
        print >> fw, last_time
        fw.close()

class ZipStorage(DirectoryStorage):
    def __init__(self, folderAndFileTemplate, notifier):
        self.setBaseDirectoryAndFragment(folderAndFileTemplate)
        self.notifier = notifier
        self._openZipFile()
        self._readDownloadedIds()
        self._readLabelAssignment()

    def setBaseDirectoryAndFragment(self, folderAndFileTemplate):
        super(ZipStorage, self).setBaseDirectoryAndFragment(folderAndFileTemplate)
        self.zipFilename = self.baseDirectory
        self.zipFilenameWithoutExtension = self.getZipFilenameWithoutExtension()
        self.baseDirectory = os.path.dirname(self.zipFilename)

    def _openZipFile(self):
        try:
            os.makedirs(self.baseDirectory)
        except OSError:
            pass

    def getZipFilenameWithoutExtension(self):
        return os.path.splitext(self.zipFilename)[0]

    def getIdCacheFilename(self):
        '''Get the full filename of the messageId cache file.
        
        If the zip file is "C:\My Folder\MyGmailBackup.zip", the messageId
        cache filename will be "C:\My Folder\MyGmailBackup.ids.txt".
        '''
        filename = self.zipFilenameWithoutExtension + '.ids.txt'
        return filename

    def getLabelsFilename(self):
        '''Get the full filename of the labels file.
        
        If the zip file is "C:\My Folder\MyGmailBackup.zip", the labels
        filename will be "C:\My Folder\MyGmailBackup.labels.txt".
        '''
        filename = self.zipFilenameWithoutExtension + '.labels.txt'
        return filename

    def getStampFilename(self):
        '''Get the full filename of the stamp file.
        
        If the zip file is "C:\My Folder\MyGmailBackup.zip", the stamp
        filename will be "C:\My Folder\MyGmailBackup.stamp.txt".
        '''
        filename = self.zipFilenameWithoutExtension + '.stamp.txt'
        return filename


    def iterateBackupFiles(self, sinceAsSecondsSinceTheEpoch=None, beforeAsSecondsSinceTheEpoch=None, logging=True):
        '''
        
        
        JH: Note that if self.zipFilename (the backup store) does not 
        exist, nothing at all will happen, not even an error message 
        to the user. I believe this should be changed.
        '''
        if os.path.exists(self.zipFilename):
            zipFileHandle = zipfile.ZipFile(self.zipFilename, 'r')
            listing = [zipInfoObject.filename for zipInfoObject in zipFileHandle.infolist()]
            # skip labels.txt and labels.txt.bak files
            for index, messageFilename in enumerate(listing):
                try:
                    messageAsString = zipFileHandle.read(messageFilename)

                    messageDateAsDateTimeTuple = getDateTimeTupleFromMessageAsString(messageAsString)
                    messageDateInSecondsSinceTheEpoch = time.mktime(messageDateAsDateTimeTuple)
                    from_address, subject = _getSenderAndSubjectFromMessageAsString(messageAsString)
                    shouldBeRestored = \
                        (sinceAsSecondsSinceTheEpoch is None or sinceAsSecondsSinceTheEpoch < messageDateInSecondsSinceTheEpoch) and \
                        (beforeAsSecondsSinceTheEpoch is None or messageDateInSecondsSinceTheEpoch < beforeAsSecondsSinceTheEpoch)
                    if shouldBeRestored:
                        yield messageFilename, messageAsString

                    restored = shouldBeRestored
                    if logging:
                        if restored:
                            self.notifier.notifyEmailRestore(from_address, subject, index+1, len(listing))
                        else:
                            self.notifier.notifyEmailRestoreSkip(from_address, subject, index+1, len(listing))
                except GeneratorExit:
                    break
                except:  # All other exceptions.
                    self.notifier.handleException(_("Error occurred while reading email from disk"))

    def storeMessage(self, messageAsString, messageObject):

        if not os.path.exists(self.zipFilename):
            fileMode = 'w'
        else:
            fileMode = 'a'

        zipFileHandle = zipfile.ZipFile(self.zipFilename, fileMode, zipfile.ZIP_DEFLATED)

        try:
            listing = zipFileHandle.namelist()

            msg_fn = self.getMessageFilename(messageAsString)
            msg_iid = _getMessageIdFromMessageObject(messageObject)
            idx = 1
            while True:
                msg_fn_num = '%s-%01d.eml'%(msg_fn, idx)
                idx += 1
                if not msg_fn_num in listing:
                    break
            zipFileHandle.writestr(msg_fn_num, messageAsString)
        finally:
            zipFileHandle.close()
            
        # Only store msg_fn_num and msg_iid if the writing to zipFileHandle above succeeded.
        self.messageIdToMessageFilename[msg_iid] = msg_fn_num
        self.messageFilenameToMessageId[msg_fn_num] = msg_iid



class GmailBackup(object):
    
    def __init__(self, username, password, notifier, lang=None):
        self.notifier = notifier
        self.username = username
        self.password = password
        self.gmailConnection = GmailConnection.GmailConnection(username, password, notifier, lang)

    def iterMails(self, where, setOfMessageIdsOfMessagesToSkip=set()):
        '''
        Yield the messageAsString for all messages that match the given 
        'where' clause and whose messageIds does not match the given list 
        of messageIds to skip.
        '''
        self.gmailConnection.selectMailboxAllMail()

        messageNumberList = self.gmailConnection.search(where)
        numberOfMessagesFound = len(messageNumberList)

        numberOfMessagesToSkip = len(setOfMessageIdsOfMessagesToSkip)
        numberOfSkippedMessages = 0

        for index, messageNumber in enumerate(messageNumberList):
            try:
                messageId = self.gmailConnection.fetchMessageId(messageNumber)
                if messageId in setOfMessageIdsOfMessagesToSkip:
                    numberOfSkippedMessages += 1
                    self.notifier.notifyEmailBackupSkip(index+1, numberOfMessagesFound, numberOfSkippedMessages, numberOfMessagesToSkip)
                else:
                    messageAsString = self.gmailConnection.fetchMessageAsString(messageNumber)
                    messageObject = _getMessageObjectFromMessageAsString(messageAsString)
                    yield messageAsString, messageObject
                    from_address, subject = _getSenderAndSubjectFromMessageAsString(messageAsString)
                    self.notifier.notifyEmailBackup(from_address, subject, index+1, numberOfMessagesFound)
            except (GeneratorExit, ):
                # The exception is a GeneratorExit exception.
                # Stop the iteration by breaking out of the for loop.
                break  
            except:
                # The exception was another exception.
                # Report it to the log, but continue the iteration
                # unless handleError() deems it fatal.
                self.notifier.handleException(_("Error occurred while downloading e-mail"))

        self.gmailConnection.close()

    def getLabels(self):
        listOfLines = self.gmailConnection.list()
        labels = []
        for line in listOfLines:
            match = re.match(r'^(\(.*\))\s".*?"\s"(.*)"\s*$', line)
            flags = match.group(1)
            if '\\HasNoChildren' in flags:
                label = match.group(2)
                if not re.match(r'^\[.*\].*$', label) and label != 'INBOX':
                    labels.append(label)
        labels.append('INBOX')
        return labels

    def getMessageIdsOfAllMessagesWithLabel(self, label, where=['ALL']):
        '''
        Return a generator iterator yielding the messageIds of all 
        messages with the given label.
        '''
        self.gmailConnection.selectMailbox(label)
        retries = 0
        searchSucceeded = False
        while not searchSucceeded:
            try:
                numberList = self.gmailConnection.search(where)  # This returns a list of numbers.
                searchSucceeded = True
            except imaplib.IMAP4.error:
                retries += 1
                if retries <= MAX_LABEL_RETRIES:
                    self.gmailConnection.selectMailbox(label)
                else:
                    self.notifier.notifyError(_("Cannot backup the assignment of label: %s") % label)
                    raise StopIteration

        for messageNumber in numberList:
            yield self.gmailConnection.fetchMessageId(messageNumber)


    def getLabelAssignment(self, where=['ALL']):

        messageIdToLabelSetDict = {}  # Keys are messageIds, values are sets of labels.

        self.gmailConnection.connect()
        
        # Iterate over all labels.
        for label in self.getLabels():
            self.notifier.notifyLog(_("Getting list of messages for label %s") % label)
            try:
                # Iterate over all messages with the given label.
                for messageId in self.getMessageIdsOfAllMessagesWithLabel(label, where):
                    if messageId not in messageIdToLabelSetDict:
                        messageIdToLabelSetDict[messageId] = set()  # No previous labels. Create new set.
                    messageIdToLabelSetDict[messageId].add(label)  # Add label to set.
            except:
                self.notifier.handleException(_("Error while doing backup of label %r") % label)
        return messageIdToLabelSetDict


    def backupLabels(self, whereList, storage):
        self.notifier.notifyLabelsBackup(False)

        messageIdToLabelSetDict = self.getLabelAssignment(whereList)
        storage.updateLabelAssignment(messageIdToLabelSetDict)
        
        self.notifier.notifyLabelsBackup(True)



    def backupMessages(self, whereList, storage, newestBackedUpEmailDateTimeTuple):

        self.notifier.notifyLog(_("Starting backup of messages"))

        self.gmailConnection.connect()

        setOfDownloadedIds = storage.getSetOfDownloadedMessageIds()
        try:
            for messageAsString, messageObject in self.iterMails(whereList, setOfDownloadedIds):
                try:
                    storage.storeMessage(messageAsString, messageObject)
                    # Update newestBackedUpEmailDateTimeTuple.
                    # JH: Isn't there a problem with this? Registering
                    # the newest backed up email doesn't make sure that
                    # all earlier emails have been backed up, because
                    # the emails might not (?) come in chronological
                    # order from the server. So if the user stops the
                    # process (or if the user only selected
                    # to backup a specific time interval), we cannot
                    # be sure that we really did back up all emails until
                    # the datetime newestBackedUpEmailDateTimeTuple.
                    messageDateTimeTuple = _getMessageDateAsDateTimeTuple(messageObject)
                    if messageDateTimeTuple > newestBackedUpEmailDateTimeTuple or newestBackedUpEmailDateTimeTuple is None:
                        newestBackedUpEmailDateTimeTuple = messageDateTimeTuple
                except:
                    self.notifier.handleException(_("Error while saving e-mail"))
        
        finally:
            storage.storeComplete()

        self.notifier.notifyLog(_("Ended backup of messages"))
        
        return newestBackedUpEmailDateTimeTuple


    def backup(self, folderAndFileTemplate, whereList=['ALL'], onlyNewest=False):
        '''Do the backup.
        
        JH: I believe the since and before dates should be arguments to this function 
        to avoid the hack below where an already existing SINCE clause is updated.
        The SINCE and BEFORE clauses should be built here instead.'''
        storage = EmailStorage.createStorage(folderAndFileTemplate, self.notifier)

        newestBackedUpEmailDateTimeTuple = storage.newestBackedUpEmailDateTimeTuple()
        
        if onlyNewest:
            if newestBackedUpEmailDateTimeTuple is not None:
                newestBackedUpEmailDateYYYYMMDD = DateTimeConverter.convertDateTimeTupleToDateYYYYMMDD(newestBackedUpEmailDateTimeTuple)
                sinceImap4InternalDateString = DateTimeConverter.convertDateYYYYMMDDToImap4InternalDateString(newestBackedUpEmailDateYYYYMMDD)
                try:
                    # Try to update an already existing SINCE clause if there is one.
                    idx = whereList.index('SINCE')
                    whereList[idx+1] = sinceImap4InternalDateString
                except ValueError:
                    # That update failed. Then there is probably no existing SINCE clause.
                    # Add a new SINCE clause.
                    whereList.append('SINCE')
                    whereList.append(sinceImap4InternalDateString)

        self.notifier.notifyVersion()
        self.notifier.notifyBackup(False, self.username, folderAndFileTemplate)

        newestBackedUpEmailDateTimeTuple = self.backupMessages(whereList, storage, newestBackedUpEmailDateTimeTuple)

        self.backupLabels(whereList, storage)

        storage.updateStamp(newestBackedUpEmailDateTimeTuple)

        self.notifier.notifyBackup(True, self.username, folderAndFileTemplate)

    
    def restoreLabels(self, assignment, sinceImap4InternalDateString, beforeImap4InternalDateString):
        self.gmailConnection.selectMailboxAllMail()

        whereList = []
        whereList.append('SINCE')
        whereList.append(sinceImap4InternalDateString)
        whereList.append('BEFORE')
        whereList.append(beforeImap4InternalDateString)

        numbers = self.gmailConnection.search(whereList)

        message_by_labels = {}
        labels = set()

        for idx, num in enumerate(numbers):
            try:
                imsg_id = self.gmailConnection.fetchMessageId(num)
                if imsg_id in assignment:
                    for label in assignment[imsg_id]:
                        if label not in message_by_labels:
                            message_by_labels[label] = []
                        message_by_labels[label].append(num)
                        labels.add(label)
                self.notifier.notifyLabelsRestore(idx+1, len(numbers))
            except:
                self.notifier.handleException(_("Error while getting MessageID"))

        for label in labels:
            try:
                message_set = message_by_labels[label]
                message_set = ','.join(message_set)
                self.gmailConnection.create(label)
                self.gmailConnection.copy(message_set, label)
            except:
                self.notifier.handleException(_("Error while restoring label %r") % label)

    def restore(self, filenamePattern, sinceDateYYYYMMDD=None, beforeDateYYYYMMDD=None):

        if sinceDateYYYYMMDD:
            sinceTimeInSecondsSinceTheEpoch = DateTimeConverter.convertDateYYYYMMDDToSecondsSinceEpoch(sinceDateYYYYMMDD)
        else:
            sinceTimeInSecondsSinceTheEpoch = None
        
        if beforeDateYYYYMMDD:
            beforeTimeInSecondsSinceTheEpoch = DateTimeConverter.convertDateYYYYMMDDToSecondsSinceEpoch(beforeDateYYYYMMDD)
        else:
            beforeTimeInSecondsSinceTheEpoch = None

        self.notifier.notifyVersion()
        self.notifier.notifyRestore(False, self.username, filenamePattern) # False means not finished.
        self.gmailConnection.connect()

        storage = EmailStorage.createStorage(filenamePattern, self.notifier)

        setOfDateTimeTuplesForAllMailsInSelectedTimePeriod = set()
        for messageFilename, messageAsString in storage.iterateBackupFiles(sinceTimeInSecondsSinceTheEpoch, beforeTimeInSecondsSinceTheEpoch):  # @UnusedVariable messageFilename
            try:
                messageTimeAsImap4InternalDate = getImap4InternalDateStringFromMessageAsString(messageAsString)
                self.gmailConnection.append(self.gmailConnection.imapFolderNameAllMail, "(\Seen)", messageTimeAsImap4InternalDate, messageAsString)

                messageTimeAsDateTimeTuple = getDateTimeTupleFromMessageAsString(messageAsString)
                setOfDateTimeTuplesForAllMailsInSelectedTimePeriod.add(messageTimeAsDateTimeTuple)
            except:
                self.notifier.handleException(_("Error while restoring e-mail")) # JH: Should add %s with email filename.

        if setOfDateTimeTuplesForAllMailsInSelectedTimePeriod:
            timeOfEarliestMailAsDateTimeTuple = min(setOfDateTimeTuplesForAllMailsInSelectedTimePeriod)
            timeOfLastMailAsDateTimeTuple = max(setOfDateTimeTuplesForAllMailsInSelectedTimePeriod)

            shiftedTimeOfEarliestMailAsDateTimeTuple, shiftedTimeOfLastMailAsDateTimeTuple = _shiftAndTrimDateTimeTuplesIfPossible(timeOfEarliestMailAsDateTimeTuple, timeOfLastMailAsDateTimeTuple)
            
            minimumDateAsImap4InternalDate = imaplib.Time2Internaldate(shiftedTimeOfEarliestMailAsDateTimeTuple)
            maximumDateAsImap4InternalDate = imaplib.Time2Internaldate(shiftedTimeOfLastMailAsDateTimeTuple)

            copyOfMessageIdToLabelsDict = storage.getCopyOfMessageIdToLabelsDict()

            self.restoreLabels(copyOfMessageIdToLabelsDict, minimumDateAsImap4InternalDate, maximumDateAsImap4InternalDate)

        self.notifier.notifyRestore(True, self.username, filenamePattern) # True means finished.


    def clear(self):
        '''
        Delete all emails and labels from the account (!).
        '''
        self.notifier.notifyVersion()
        self.notifier.notifyClear(False, self.username)

        self.gmailConnection.connect()

        # Delete all mails in a normal search.
        self.gmailConnection.selectMailboxAllMail()
        data = self.gmailConnection.search(['ALL'])
        nums = ','.join(data)
        if nums:
            self.gmailConnection.copy(nums, self.gmailConnection.imapFolderNameTrash)
            self.gmailConnection.store(nums, 'FLAGS.SILENT', '\\Deleted')
            self.gmailConnection.expunge()

        # Now delete all mails in the Trashcan.
        self.gmailConnection.selectMailbox(self.gmailConnection.imapFolderNameTrash)
        data = self.gmailConnection.search(['ALL'])
        nums = ','.join(data)
        if nums:
            self.gmailConnection.store(nums, 'FLAGS.SILENT', '\\Deleted')
            self.gmailConnection.expunge()

        # Now delete all labels.
        for label in self.getLabels():
            self.gmailConnection.delete(label)

        self.gmailConnection.close()

        self.notifier.notifyClear(True, self.username)


    def list(self):
        '''
        For each mailbox, yield a pair of the mailbox name in IMAP format and 
        the number of messages in the mailbox. 
        
        If there is an error finding the number of messages for a specific 
        mailbox, return -1 as the number of messages for that mailbox.
        '''
        self.gmailConnection.connect(guessLanguage=False)

        linesInImapFormatList = self.gmailConnection.list()

        for line in linesInImapFormatList:
            matchObject = re.match(r'^\(.*\)\s".*"\s"(.*)"\s*$', line)
            mailboxNameInImapFormat = matchObject.groups(1)[0]

            self.gmailConnection.selectMailbox(mailboxNameInImapFormat)
            try:
                data = self.gmailConnection.search(['ALL'])
                numberOfMessages = len(data)
            except imaplib.IMAP4.error:
                numberOfMessages = -1
            yield mailboxNameInImapFormat, numberOfMessages

    def getNewestVersionAvailable(self):
        '''
        Return the revision number (as an integer number) of the newest
        version of Gmail Backup available online.
        
        Return None if the newest version cannot be found.
        '''
        try:
            with urllib.urlopen(VERSION_URL) as webPageStreamHandle:
                webPageContents = webPageStreamHandle.read()
                versionMatch = re.search('<td class="id"><a href=".*">r(\d+)</a></td>', webPageContents)
                if versionMatch:
                    try:
                        newestAvailableRevisionNumber = int(versionMatch.group(1))
                    except ValueError:
                        # Couldn't convert the found digits to an integer.
                        # This should be virtually impossible, as Python has long
                        # integers of virtually any size. I guess only memory
                        # puts a limit to it, and if it does, maybe the error
                        # raised is not even a ValueError but a MemoryError.
                        newestAvailableRevisionNumber = None
                else:
                    # Couldn't find version info on the page.
                    newestAvailableRevisionNumber = None
        except:
            # Couldn't open url.
            newestAvailableRevisionNumber = None

        return newestAvailableRevisionNumber


    def getNewVersionInfo(self, currentRevisionNumber=GMB_REVISION):
        '''
        Return a pair with (revision, url), where revision is the newest 
        revision available online, and url is the URL where it is 
        available.
        
        If there is no new version available or if information about
        it cannot be found due to network problems or other problems, 
        return (None, None).
        
        JH: Note that this by default compares the version on the web with
        the revision GMB_REVISION. This does not take into account
        if the GUI version or the command line version has changed – 
        only whether the core gmb.py has changed. I would call this 
        a bug. (2013-08-20)
        '''
        newestAvailableRevisionNumber = self.getNewestVersionAvailable()
        if newestAvailableRevisionNumber:
            if newestAvailableRevisionNumber > currentRevisionNumber:
                newRevisionAndUrl = (newestAvailableRevisionNumber, DOWNLOAD_URL)
            else:
                newRevisionAndUrl = (None, None)
        else:
            newRevisionAndUrl = (None, None)

        return newRevisionAndUrl

    
    def checkForNewVersion(self, currentRevisionNumber=GMB_REVISION):
        '''
        
        :param currentRevisionNumber:
        :type currentRevisionNumber:
        
        JH: Note that this by default compares the version on the web with
        the revision GMB_REVISION. This does not take into account
        if the GUI or the command line versions have changed – 
        only whether the core gmb.py has changed. I would call this 
        a bug. (2013-08-20)
        '''
        newRevisionNumber, url = self.getNewVersionInfo(currentRevisionNumber)
        if newRevisionNumber:
            msg = _(
                'New version of Gmail Backup is available!\n'
                'You can download version %s here:\n'
                '%s'
                ) % (newRevisionNumber, url)
        else:
            # JH: Note that this does not necessarily mean that the user
            # is using the latest version. It could also be that the 
            # fetching of the latest available version info failed,
            # for instance due to network problems or if the web site
            # is down.
            # This message should only be given if the latest version
            # available was positively found and is equal to the current
            # version.
            msg = _("You are using the latest version of Gmail Backup.")
        self.notifier.notifyLog(msg)
        return newRevisionNumber
